#!/usr/bin/env python

'''
http://dict.cn is an online dictionary I often use to find meanings of unknown words.
It provides a service that I can store all the words I query for some days I will
review and try to remember them. I found that it is a easy way for English learners
to enlarge the number of known words.

Tested on Windows.
'''

import urllib
import urllib2
import cookielib
import re
import codecs
import sys
import getopt

class DictNet:
    '''This utility class helps you to login dict.cn and fetch your vocabulary with
    explanations and store them into a text file for other use. It also can be use
    to search meanings of individual words.'''
    
    site_url = 'http://bbs.dict.cn'
    DICT_SHOW_RE=re.compile(r'javascript:dictShow\("([\w ]+)"\)')
    VOC_NEXT_RE=re.compile(r'href="/vocabulary\.php(\?range_method=[^"]+)"')
    
    def __init__(self, username = None, password = None):
        self.username = username
        self.password = password
        self.Reader = codecs.getreader('gb2312')
        
    def login(self):
        '''login dict.cn and store cookies for further operations'''
        
        cj = cookielib.CookieJar()
        opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj),
                                      urllib2.HTTPRedirectHandler())
        urllib2.install_opener(opener)
        login_url = DictNet.site_url + '/login.php'
        login_data = urllib.urlencode({'username':self.username,
                                       'password':self.password,
                                       'autologin':'on',
                                       'login':'login' # this is required for login, maybe by backend php
                                       })
        try:
            # get session id first
            urllib2.urlopen( login_url )

            # do login            
            result = urllib2.urlopen( login_url, login_data ).read()
        except Exception, e:
            print 'Exception: ' + e
            return False

        # check the page returned after logined
        if result.find(self.username) >= 0 :
            return True
        else:
            return False
        
    def logout(self):
        pass
    
    def get_vocabulary(self):
        '''get my vocabulary list from dict.cn'''
        print 'Fetching vocabulary:',

        url = DictNet.site_url + '/vocabulary.php'
        voc=set()

        # read the first page
        cont = urllib2.urlopen(url).read()

        # get the url parameters of other vocabulary pages
        pages=[]
        for match in DictNet.VOC_NEXT_RE.finditer(cont):
            p = match.group(1)
            if p not in pages:
                pages.append(p)

        # read the words on the frist page        
        for match in DictNet.DICT_SHOW_RE.finditer(cont):
            w = match.group(1)
            voc.add(w)
            print '.',

        # read the words on the other pages
        for p in pages:
            cont = urllib2.urlopen(url+p.replace('&amp;','&')).read()
            for match in DictNet.DICT_SHOW_RE.finditer(cont):
                w = match.group(1)
                voc.add(w)
                print '.',

        print 'Done'
        print 'Totol: %d' % len(voc)
        return voc

    def search_word(self, word):
        '''search explanations of a give word'''
        print 'Searching word "%s" ' % word ,
        try:
            url = 'http://dict.cn/search/?q='+re.sub('\s+','+',word)

            # using 'replace' error handling scheme            
            cont = self.Reader(urllib2.urlopen(url), 'replace').read()

            # get the position of explanations of the word            
            start = cont.find('<big><font size="2" face="Trebuchet MS">')+len('<big><font size="2" face="Trebuchet MS">')
            end = cont.find('</big>', start)

            # for multiple meanings
            exps = cont[start:end].splitlines()
            exps = [e.replace('<br>', '') for e in exps]
            print
            return exps
        except Exception,e:
            print 'Failed.'
            print e
            return None
        
    def get_word_list(self):
        '''fetch my vocabulary with explanations from dict.cn'''
        word_list = {}
        voc = self.get_vocabulary()
        for w in voc:
            if w in word_list: continue
            m = self.search_word(w)
            if m: word_list[w] = m
        return word_list

    def save(self, filename, wordlist):
        '''save wordlist into a file'''
        f=codecs.getwriter('utf_8')(open(filename, 'w'))
        for word in wordlist:
            f.write(word+'\r\n')
            for m in wordlist[word]:
                f.write('\t'+m+'\r\n')
        f.close()
        print 'Saved wordlist into %s' % filename

    def clear(self):
        '''clear my vocabulary on dict.cn for a new starting'''
        pass
        
def gen_word_list(username, password, filename):
    wn = DictNet(username, password)
    if wn.login():
        print 'Logined'
        wordlist =  wn.get_word_list()
        wn.save(filename, wordlist)
    else:
        print 'Login Failed'

def test():
    wn = DictNet()
    m = wn.search_word('vein')
    wl = {'vein':m}
    #wn.save('wordlist.txt', wl)

def main():
    def usage(prog):
        print 'Fetch vocabulary from dict.cn.'
        print '%s -u username -p password -o output_filename' % prog
        
    # parse arguments
    try:
        opts,args = getopt.getopt(sys.argv[1:], 'u:p:o:')
    except getopt.GetoptError,e:
        print usage(sys.argv[0])
        print str(e)
        sys.exit(2)
    
    username = None
    password = None
    output = None
    
    for o, a in opts:
        if o=='-u':
            username = a
        elif o=='-p':
            password = a
        elif o=='-o':
            output = a
        else:
            usage()
            print 'Invalid option: ' + o
            sys.exit(0)

    gen_word_list(username, password, output)
        
if __name__=='__main__':
    #test()
    main()
